#coding:utf-8
import re,os,sys
sys.path.append("matchutils")
from config import *
import pygtrie 
from langconv import *
import datetime

not_names = {'未知':1, '暂无':1, '无':1, '其它':1,'其他':1, '缺':1, '中国':1, '中国内地':1, '内地':1, '台湾':1, '香港':1, 'editors':1, 'editor':1, '北京':1, '不详':1, '天津':1, '上海':1, '浙江':1, '其他栏目':1, '其他频道':1, '台湾':1, '日本':1, '淘气':1}

#今年
cur_year = datetime.datetime.now().strftime('%Y')

dig_map = {}
dig_map[u'1'] = u'一'
dig_map[u'2'] = u'二'
dig_map[u'3'] = u'三'
dig_map[u'4'] = u'四'
dig_map[u'5'] = u'五'
dig_map[u'6'] = u'六'
dig_map[u'7'] = u'七'
dig_map[u'8'] = u'八'
dig_map[u'9'] = u'九'
dig_map[u'0'] = u'零'
dig_map[u'１'] = u'一'
dig_map[u'２'] = u'二'
dig_map[u'３'] = u'三'
dig_map[u'４'] = u'四'
dig_map[u'５'] = u'五'
dig_map[u'６'] = u'六'
dig_map[u'７'] = u'七'
dig_map[u'８'] = u'八'
dig_map[u'９'] = u'九'
dig_map[u'０'] = u'零'


fame_name = {}
fame_name['小'] = 1
fame_name['老'] = 1
fame_name['张'] = 1
fame_name['李'] = 1
fame_name['赵'] = 1
fame_name['王'] = 1
fame_name['陈'] = 1
fame_name['钱'] = 1
fame_name['孙'] = 1
fame_name['周'] = 1
fame_name['吴'] = 1
fame_name['刘'] = 1
fame_name['郑'] = 1
fame_name['冯'] = 1
fame_name['杨'] = 1
fame_name['蒋'] = 1
fame_name['朱'] = 1
fame_name['韩'] = 1
fame_name['沈'] = 1
fame_name['秦'] = 1
fame_name['何'] = 1
fame_name['施'] = 1
fame_name['吕'] = 1
fame_name['黄'] = 1
fame_name['窦'] = 1
fame_name['苏'] = 1
fame_name['马'] = 1
fame_name['宋'] = 1
fame_name['范'] = 1
fame_name['葛'] = 1
fame_name['袁'] = 1
fame_name['顾'] = 1
fame_name['柳'] = 1
fame_name['鲁'] = 1
fame_name['彭'] = 1
fame_name['唐'] = 1
fame_name['康'] = 1
fame_name['傅'] = 1
fame_name['于'] = 1
fame_name['肖'] = 1
fame_name['萧'] = 1
fame_name['阮'] = 1
fame_name['戴'] = 1
fame_name['邵'] = 1
fame_name['姚'] = 1
fame_name['董'] = 1
fame_name['杜'] = 1
fame_name['贺'] = 1
fame_name['薛'] = 1
fame_name['胡'] = 1
fame_name['霍'] = 1
fame_name['梁'] = 1
fame_name['曹'] = 1

han_dig_map = {}
han_dig_map[u'零'] = 0
han_dig_map[u'一'] = 1
han_dig_map[u'二'] = 2
han_dig_map[u'两'] = 2
han_dig_map[u'三'] = 3
han_dig_map[u'四'] = 4
han_dig_map[u'五'] = 5
han_dig_map[u'六'] = 6
han_dig_map[u'七'] = 7
han_dig_map[u'八'] = 8
han_dig_map[u'九'] = 9
han_dig_map[u'十'] = 10
han_dig_map[u'十一'] = 11
han_dig_map[u'十二'] = 12
han_dig_map[u'十三'] = 13
han_dig_map[u'十四'] = 14
han_dig_map[u'十五'] = 15
#han_dig_map[u'十'] = 10

def isNumberStr( s ):
    if s.isdigit():
        return True
    if isinstance(s, str):
        s = s.decode('utf-8', 'ignore')
    if not isinstance(s, unicode):
        return False
    for ch in s:
        if ch in han_dig_map:
            continue
        if ch == u'十':
            continue
        if ch == u'百':
            continue
        if ch == u'千':
            continue
        if ch == u'万':
            continue
        return False
    return True


def fromHanDigit( line ):
    oldline = line
    if isinstance( line, str):
        line = line.decode('utf-8', 'ignore')

    if line.isdigit():
        return line.encode('utf-8', 'ignore')

    if line == u'零零七':
        return '007'

    if line == u'百':
        return '百'
    if line == u'千':
        return '千'
    if line == u'万':
        return '万'
    if line == u'百万':
        return '百万'

    if not isinstance(line, unicode):
        return oldline

    if line.find(u'亿') != -1:
        return oldline
    if line.find(u'万') != -1:
        return oldline

    l = []
    num = len( line )
    big = 0
    result = 0
    for i in range(num):
        ch = line[i]
        if ch in han_dig_map and ch != u'十':
            result = result * 10 + han_dig_map[ ch ]
            continue
        if ch == u'千':
            result *= 1000 
            if result == 0:
                result = 1000
            big += result
            result = 0
            continue
        if ch == u'百':
            result *= 100 
            if result == 0:
                result = 100
            big += result
            result = 0
            continue
        if ch == u'十':
            result *= 10 
            if result == 0:
                result = 10
            big += result
            result = 0
            continue
        return oldline

    result = result + big
    return str(result)

#繁体简体
def simple2tradition(line):
    if not isinstance(line, str):
        return line
    #将简体转换成繁体
    line = Converter('zh-hant').convert(line.decode('utf-8'))
    line = line.encode('utf-8')
    return line

def tradition2simple(line):
    if not isinstance(line, str):
        return line
    # 将繁体转换成简体
    line = Converter('zh-hans').convert(line.decode('utf-8'))
    line = line.encode('utf-8')
    return line

def is_unicode_hanzi(c):
    d = ord(c)
    #基本汉字
    if d >= 0x4E00 and d <= 0x9FA5:
        return True
    #补充汉字
    if d >= 0x9FA6 and d <= 0x9FCB:
        return True
    #扩展A
    if d >= 0x3400 and d <= 0x4DB5:
        return True
    #兼容汉字
    if d >= 0xF900 and d <= 0xFAD9:
        return True
    return False

def is_all_hanzi( line ):
    if isinstance( line, str):
        line = line.decode('utf-8', 'ignore')
    if not isinstance( line, unicode):
        return False
    for ch in line:
        if not is_unicode_hanzi(ch):
            return False
    return True

def is_english(ch):
    if ord(ch) >= ord(u'a') and ord(ch) <= ord(u'z'):
        return True
    if ord(ch) >= ord(u'A') and ord(ch) <= ord(u'Z'):
        return True
    return False

def is_english_line( line ):
    if isinstance(line,str):
        line = line.decode('utf-8', 'ignore')
    if not isinstance(line, unicode):
        return False
    eng = 0
    for ch in line:
        if ch == u'-' or ch == u'·': continue
        if ch == u'—' : continue
        if is_unicode_hanzi( ch ):
            return False
        if is_english(ch):
            eng += 1
    if eng >= 3:
        return True
    return False

def half2full( s ):
    """半角转全角"""
    if not isinstance(s, str):
        return s
    ustring = s.decode('utf-8', 'ignore')
    rstring = ""
    for uchar in ustring:
        inside_code=ord(uchar)
        if inside_code == 32:                                 #半角空格直接转化                  
            inside_code = 12288
        elif inside_code >= 32 and inside_code <= 126:        #半角字符（除空格）根据关系转化
            inside_code += 65248

        rstring += unichr(inside_code)
    return rstring

def full2half(s):
    if not isinstance(s, str):
        return s

    ustring = s.decode('utf-8', 'ignore')
    rstring = u""
    for uchar in ustring:
        inside_code=ord(uchar)
        if inside_code == 12288:                                 #半角空格直接转化                  
            inside_code = 32
        elif inside_code >= 65280 and inside_code <= 65374:        #半角字符（除空格）根据关系转化
            inside_code -= 65248
        rstring += unichr(inside_code)
    rstring = rstring.encode('utf-8', 'ignore')
    return rstring

import re


#语言
def deal_lang( line ):
    if isinstance(line, str):
        line = line.decode('utf-8', 'ignore')

    #语言
    lang = []
    r = re.search(u'[\(\[（【][ ]?[美俄华英法德日韩汉国泰粤外台][文语]?[ ]?[\]】）\)]', line )
    if r is not None:
        w = r.group()
        line = line.replace(w, u'').strip()
        w = w[1:-1].encode('utf-8', 'ignore').strip()
        lang.append( w )
        return line, lang

    #结尾
    r = re.search(u' [美台华俄国日韩汉国英法德泰华粤外][文语]$', line )
    if r is not None:
        line = line[:-3]
        w = r.group()
        w = w.encode('utf-8', 'ignore').strip()
        lang.append( w )
        return line, lang

    #任意位置
    r = re.search(u'[美台日韩汉国英法德泰华粤外][文语]版', line )
    if r is not None:
        w = r.group()
        lang.append( w.encode('utf-8', 'ignore') )
        line = line.replace(w, u'').strip()
        return line, lang

    return line, lang


def deal_year( line ):
    if isinstance(line, str):
        line = line.decode('utf-8', 'ignore')
    year = []
    #任意
    r = re.search(u'[^0-9]([12][09][0-9][0-9]年[版]?)', line )
    if r is not None:
        w = r.groups()[0]
        line = line.replace(w, u'').strip()
        w = w.replace(u'年', u'').replace(u'版',u'')
        year.append(w.encode('utf-8', 'ignore') ) 
        return line, year

    #末尾
    r = re.search(u'[^0-9](20[0-9][0-9])$|[^0-9](19[3456789][0-9])$', line )
    if r is not None:
        w, b = r.groups()
        if w is None:
            w = b
        line = line[:-len(w)]
        year.append(w.encode('utf-8', 'ignore') ) 
        return line, year

    #开头
    r = re.search(u'^(20[012][0-9][年 ])..', line ) 
    if r is not None:
        w = r.groups()[0]
        num = len(w)
        w = w.replace(u'年', u'').replace(u'版',u'').strip()
        if w >= "1970" and w <= cur_year:
            year.append(w.encode('utf-8', 'ignore') ) 
            line = line[num:]
        return line, year

    #开头
    r = re.search(u'^(19[3456789][0-9][年 ])..', line) 
    if r is not None:
        w = r.groups()[0]
        line = line[len(w):]
        w = w.replace(u'年', u'').replace(u'版',u'')
        year.append(w.encode('utf-8', 'ignore') ) 
        return line, year

    #任意
    r = re.search(u'[^0-9]([12][09][0-9][0-9]版)', line )
    if r is not None:
        w = r.groups()[0]
        line = line.replace(w, u'').strip()
        w = w.replace(u'年', u'').replace(u'版',u'')
        year.append(w.encode('utf-8', 'ignore') ) 
        return line, year

    #任意 
    r = re.search(u'[【（\[\(]19[3456789][0-9][年版]?[\)\]）】]', line ) 
    if r is not None:
        w = r.group()
        line = line.replace(w, u'').strip()
        w = w.replace(u'年', u'').replace(u'版',u'')
        w = w[1:-1]
        year.append(w.encode('utf-8', 'ignore') ) 

    #任意 
    r = re.search(u'[【（\[\(]\[20[012][0-9][年版]?[\)）】\]]', line ) 
    if r is not None:
        w = r.group()
        line = line.replace(w, u'').strip()
        w = w.replace(u'年', u'').replace(u'版',u'')
        w = w[1:-1]
        year.append(w.encode('utf-8', 'ignore') ) 
        return line, year

    #两位数年份
    r = re.search(u'^[0-9]{2,2}[年]版', line)
    if r is not None:
        w = r.group()
        line = line[len(w):]
        w = w[:2].encode('utf-8', 'ignore')
        if w[0] == '0' or w[0] == '1':
            w = '20'+w
        else:
            w = '19'+w
        if w >= "1970" and w <= cur_year:
            year.append( w ) 
            return line, year

    r = re.search(u' [0-9]{2,2}[年]版$', line)
    if r is not None:
        w = r.group()
        line = line[:-len(w)]
        w = w.strip()
        w = w[:2].encode('utf-8', 'ignore')
        if w[0] == '0' or w[0] == '1':
            w = '20'+w
        else:
            w = '19'+w
        if w >= "1970" and w <= cur_year:
            year.append( w ) 
            return line, year

    return line, year

def deal_days(line):
    if isinstance(line, str):
        line = line.decode('utf-8', 'ignore')
    days = []
    r = re.search(u'[^0-9]([21][09][012789][0-9][01][0-9][0-3][0-9][)）]?$)', line )
    if r is not None:
        w = r.groups()[0]
        days.append( w.encode('utf-8', 'ignore') )
        line = line[:-len(w)].strip()
        return line, days
    # 陕西卫视跨年晚会 2014-12-01
    r = re.search(u'[^0-9]([12][90][0-9]{2,2}-[01][0-9]-[0123][0-9])[^0-9]', line )
    if r is not None:
        w = r.groups()[0]
        days.append(w.encode('utf-8', 'ignore'))
        line = line.replace(w, u'',1)
        return line, days
        pass
    r = re.search(u'[^0-9]([12][90][0-9]{2,2}-[01][0-9]-[0123][0-9])$', line )
    if r is not None:
        w = r.groups()[0].strip()
        days.append(w.encode('utf-8', 'ignore'))
        line = line.replace(w, u'',1)
    return line, days

def deal_part( name ):
    if isinstance(name, str):
        name = name.decode('utf-8', 'ignore')

    r = re.search(u'第[0-9零一二三四五六七八九十]+[部季期辑篇届话]', name )
    part = []
    if r is not None:
        w = r.group()
        name = name.replace( w, u'').strip()
        part.append( w.encode('utf-8', 'ignore') )
        return name, part

    #末尾一位数字
    if len(name) >= 3:
        if name[-1].isdigit() and is_unicode_hanzi(name[-2]) and is_unicode_hanzi( name[-3] ):
            part.append( name[-1].encode('utf-8', 'ignore') )
            name = name[:-1]
            return name, part

    #末尾两位数字
    if len(name) >= 4:
        if name[-1].isdigit() and name[-2] == u'1' and is_unicode_hanzi(name[-3]) and is_unicode_hanzi( name[-4] ) :
            part.append( name[-2:].encode('utf-8', 'ignore') )
            name = name[:-2]
            return name, part

    #末尾带括号数字
    r = re.search(u'[\(（【][ ]?[0-1]?[0-9][ ]?[\)】）]$', name)
    if r is not None:
        w = r.group()
        name = name[:-len(w)]
        w = w[1:-1].strip()
        part.append( w.encode('utf-8', 'ignore') )
        return name, part

    #...汉字+数字+冒号+汉字...
    fields = name.split(':')
    if len(fields) != 2:
        return name, part

    w1 = fields[0].strip()
    w2 = fields[1].strip()
    if len(w2) <= 1:
        return name, part
    if len(w1) < 3:
        return name, part
    if not is_unicode_hanzi( w2[0]):
        return name, part
    if not w1[-1].isdigit():
        return name, part
    if not is_unicode_hanzi(w1[-2]):
        return name, part

    name = w1[:-1] + " " + w2
    part.append( w1[-1].encode('utf-8', 'ignore') ) 

    return name, part


def deal_country( name ):
    area_map = {}
    area_map['泰'] = '泰国'
    area_map['台'] = '台湾'
    area_map['港'] = '香港'
    area_map['美'] = '美国'
    area_map['英'] = '英国'
    area_map['日'] = '日本'
    area_map['韩'] = '韩国'
    area_map['法'] = '法国'
    area_map['德'] = '德国'
    area_map['意'] = '意大利'
    area_map['俄'] = '俄国'
    area_map['加'] = '加拿大'
    area_map['奥'] = '奥地利'
    area_map['澳'] = '澳大利亚'
    area_map['印'] = '印度'

    area_dic = {}

    if name.startswith(u'[内地]'):
        name = name[4:]
        area_dic['内地'] = 1
    elif name.startswith(u'[大陆]'):
        name = name[4:]
        area_dic['内地'] = 1
    elif name.startswith(u'国剧'):
        area_dic['国剧'] = 1

    #开头
    r = re.search(u'^[奥澳印加法德英美日韩泰][国]?[剧影]', name)
    if r:
        w = r.group()
        name = name[len(w):].strip()
        w = w[1] + w[-1]
        w = w.encode('utf-8', 'ignore')
        area_dic[w] = 1
        return name, area_dic.keys()

    r = re.search(u' [澳奥印法德英美日韩泰][国]?[剧影]$', name)
    if r:
        w = r.group()
        name = name[:-len(w)].strip()
        w = w[1] + w[-1]
        w = w.encode('utf-8', 'ignore')
        area_dic[w] = 1
        return name, area_dic.keys()

    #任意
    r = re.search(u'([【（\(][0-9][0-9][ ]?[印加俄台港泰德法意日韩美英][影剧][\)）】]).+', name)
    if r:
        w = r.groups()[0]
        name = name.replace(w, u'')
        w = w.encode('utf-8', 'ignore')
        area_dic[w] = 1
        return name, area_dic.keys()

    #结尾
    r = re.search(u'[法德英美日韩泰俄]国版[本]?', name)
    if r:
        w = r.group()
        name = name.replace(w, u'')
        w = w[:2]
        w = w.strip()[0].encode('utf-8', 'ignore')
        w = area_map[w] 
        area_dic[w] = 1

    return name, area_dic.keys()


def is_video_part( word ):
    if isinstance(word, str):
        word = word.decode('utf-8', 'ignore')
    if not isinstance(word, unicode):
        return False
    r = re.search(u'第[0-9一二三四五六七八九十]+[季部辑话]', word )
    if r is not None:
        return True
    return False


def is_ignore( name ):
    if name.find('最新资讯') != -1:
        return True
    if name.find('演唱会') != -1:
        return True
    if name.find('最新报道') != -1:
        return True
    if name.find('最新动态') != -1:
        return True
    if name.find('最新资讯') != -1:
        return True
    if name.find('最新资讯') != -1:
        return True
    if name.find('相关报道') != -1:
        return True
    if name.find('视频报道') != -1:
        return True
    if name.find('相关歌曲') != -1:
        return True
    if name.find('相关音乐') != -1:
        return True
    if name.find('相关视频') != -1:
        return True
    if name.find('音乐视频') != -1:
        return True
    if name.find('音乐作品') != -1:
        return True
    if name.find('知识') != -1:
        return True
    if name.endswith('新闻'):
        return True
    if name.find('音乐特辑') != -1:
        return True
    if name.find('音乐专辑') != -1:
        return True
    if name.find('新闻资讯') != -1:
        return True
    #
    name = name.decode('utf-8', 'ignore')
    if re.search(u'第[0-9一二三四五六七八九十百]+[集回]', name):
        return True
    return False

def format_peoplename( name ):
    if not isinstance(name, unicode) and not isinstance(name, str):
        return []

    if isinstance(name, unicode):
        name = name.encode('utf-8', 'ignore')

    name = name.replace('-', ' ').replace('·', ' ').strip()

    #name = full2half( name )
    name = name.lower()

    if isinstance(name, str):
        name = name.decode('utf-8', 'ignore')

    #去空格
    tmp = name.replace(u' ', u'')
    if len(tmp) <= 1:
        return []

    #两个字人名
    if len(tmp) == 2:
        tmp = tmp.encode('utf-8', 'ignore')  
        name = name.encode('utf-8', 'ignore')
        if len(name) == len(tmp):
            return [ name ]
        return [ name, tmp ]

    #根据空格分割
    parts = []
    fields = name.split()
    for field in fields:
        field = field.strip()
        if field == '': continue
        parts.append(field)
    #无
    if len(parts) == 0:
        return []

    #一个域
    if len(parts) == 1:
        name = name.encode('utf-8', 'ignore')
        return [name]
    #两个域
    if len(parts) == 2:
        name = name.encode('utf-8', 'ignore')
        result = [ name ]
        t = name.replace(' ', '') 
        result.append( t )
        a = parts[0].encode('utf-8', 'ignore')
        b = parts[-1].encode('utf-8', 'ignore')
        if len(a) >= 6:
            result.append( a ) 
        if len(b) >= 6:
            result.append( b ) 
        return result

    #三个域及以上
    if len(parts) > 2:
        name = name.encode('utf-8', 'ignore')
        result = [ name ]
        t = name.replace(' ', '') 
        result.append( t )
        a = parts[0].encode('utf-8', 'ignore')
        b = parts[-1].encode('utf-8', 'ignore')
        if len(a) >= 6:
            result.append( a ) 
        if len(b) >= 6:
            result.append( b ) 
        if len(a) + len(b) >= 12:
            result.append( a + b)
            result.append( a + " " + b)
        
    return []

#语言工具
class LangTool(object):
    _lang = None
    def __init__(self):
        self._lang = {}
        fname = "lang_map.txt"
        if dict_dir != "":
            fname = dict_dir + "/lang_map.txt"

        f = open(fname, 'rb')
        self._lang = {}

        for line in f:
            line = line.strip()
            if line == '': continue
            fields = line.split('\t')
            if len(fields) != 2:
                continue
            self._lang[ fields[0] ] = fields[1]
        f.close()

        self._dic = pygtrie.CharTrie()

        self._dic[u'中国内地'] = '中文'
        self._dic[u'中国大陆'] = '中文'
        self._dic[u'内地'] = '中文'
        self._dic[u'大陆'] = '中文'
        self._dic[u'美国'] = '英语'
        self._dic[u'英国'] = '英语'
        self._dic[u'欧洲大陆'] = '英语'
        self._dic[u'日本'] = '日语'
        self._dic[u'韩国'] = '韩语'
        self._dic[u'泰国'] = '泰语'
        self._dic[u'德国'] = '德语'
        self._dic[u'法国'] = '法语'
        self._dic[u'意大利'] = '意大利语'
        self._dic[u'俄语'] = '俄国'

    def exists(self, la):
        if la in self._lang and self._lang[la] != "暂无":
            return True
        return False

    def is_lang_word(self, word ):   
        if isinstance(word, str):
            word = word.decode('utf-8', 'ignore')
        if not isinstance(word, unicode):
            return False
        word = word.replace(u'版本', u'').replace(u'版', '').strip()
        if len(word) <= 1:
            return False

        if word[-1] == u'语':
            return True

        if word == u'中文':
            return True
        if word == u'英文':
            return True
        if word == u'俄文':
            return True
        if word == u'日文':
            return True
        if word == u'普通话':
            return True

        return False
 
    def get_lang_map(self, la):
        if not la:
            return ''

        if la in self._lang:
            return self._lang[la]

        if la == '':
            return ''
        if la== "无": return ''
        if la== "未知": return ''
        if la== "暂无": return ''
        if la== "暂缺": return ''
        if la== "其它": return ''
        if la== "语言": return ''
        if la== "其他": return ''
        if la== "其他方言": return ''
        if la== "languages": return ''
        if la== "language": return ''
        tmplangs = [ la ]

        langs = la.split()

        for la in langs:
            if la in self._lang:
                tmplangs.append( self._lang[la] )
                continue
            la = la.replace('原声', '')
            if la in self._lang:
                tmplangs.append( self._lang[la] )
                continue

            pos = la.find('语')
            if pos > 0:
                la = la[:pos+3]
            if la in self._lang:
                tmplangs.append( self._lang[la] )
                continue
            if la.endswith('方言') and len(la) == 12:
                la = la.replace('方言', '话')
            tmplangs.append( la )
        tmplangs = list(set(tmplangs))
        la = ','.join( tmplangs )
        return la

    def infer_from_area(self, areastr ):
        areastr = areastr.decode('utf-8', 'ignore')
        num = len( areastr)
        langs = []
        for i in range(num):
            res, val = self._dic.longest_prefix( areastr[i:])
            if res is None:
                i += 1
                continue
            i += len(res) 
            langs.append( val )

        return langs
    def infer_from_pubstr(self, pubstr): 
        areastr = pubstr.decode('utf-8', 'ignore')
        num = len( areastr)
        langs = []
        for i in range(num):
            res, val = self._dic.longest_prefix( areastr[i:])
            if res is None:
                i += 1
                continue
            i += len(res) 
            langs.append( val )

        return langs
        pass

    def format_langstr(self, lang):
        #格式化语言
        if isinstance(lang, unicode):
            lang = lang.encode('utf-8', 'ignore')
        if not isinstance(lang, str):
            return []
        lang = lang.lower()
        langs = lang.replace('/',';').replace('|', ',').replace('、',',').replace(';',',').replace(',',',')
        langs = langs.replace("(", ',').replace(")", ",").replace('–', ',').split(',')
        tmplangs = [ self.get_lang_map( la ) for la in langs if la !='' ]
        langs = []
        for la in tmplangs:
            la = la.strip()
            if la== "无": continue
            if la== "未知": continue
            if la== "暂无": continue
            if la== "暂缺": continue
            if la== "其它": continue
            if la== "语言": continue
            if la== "其他": continue
            if la== "其他方言": continue
            if la== "languages": continue
            if la== "language": continue
            if la== "": continue
            langs.append( la )

        results = []
        for la in langs:
            results += la.split(',')
        return results

#区域工具
class AreaTool(object):
    def __init__(self):
        fname = "area_map.txt"
        if dict_dir != "":
            fname = dict_dir + "/area_map.txt"
        f = open(fname, 'rb')
        self._area = {}
        for line in f:
            line = line.strip()
            if line == '': continue
            fields = line.split('\t')
            if len(fields) != 2:
                continue
            self._area[ fields[0] ] = fields[1]
        f.close()

    def get_area_map(self, ar ):
        if ar in self._area:
            return self._area[ar]
        return ar

    def exists(self, ar ):
        if ar in self._area:
            return True
        return False

    def infer_from_lang(self, langstr):
        pass
    def infer_from_pubstr(self, pubstr): 
        pass

    def format_areastr(self, area ):
        if isinstance(area, unicode):
            area = area.encode('utf-8', 'ignore')

        if not isinstance(area, str):
            return []
        area = area.lower()
        area = area.replace(';',',').replace('地区','').replace('国家','').replace('|', ',').replace("'", '')
        areas = area.replace('区域','').replace('其它','').replace('其他','').split(',')
        #areas = [ self.get_area_map( ar ) for ar in areas if ar != '' ]

        tmpareas = []
        for ar in areas:
            ar = ar.strip()
            if ar == "": continue
            if ar == "待定": continue
            if ar == '未知' or ar == '暂无' or ar == '全部' or ar == '暂缺': 
                continue
            if ar in self._area:
                tmpareas.append( self._area[ar] )
                continue
            tmpareas.append( ar )
            ars = ar.split()
            for ar2 in ars:
                if ar2 in self._area:
                    tmpareas.append( self._area[ar2] )

        results = []
        for ar in tmpareas:
            results += ar.split(',')
        return results


def conv_string(line):
    """繁简体 全半角　大小写"""
    if line is None:
        return ''
    
    if isinstance(line, unicode):
        line = line.encode('utf-8', 'ignore')
    if isinstance(line, str):
        line = line.lower()
    return line

    #if not isinstance(line, str):
    #    return line

    #line = tradition2simple( line )
    #line = full2half(line)
    #line = line.lower().strip()

    #return line



#语言工具
langTool  = LangTool()
#区域工具
areaTool = AreaTool()


def split_get_words ( line ):
    if isinstance( line, str):
        line = line.decode('utf-8', 'ignore')
    if not isinstance( line, unicode):
        return []
    num = len( line )
    word, result, flag = [], [], 0
    for i in range( num):
        ch = line[i]
        if is_english( ch ):
            ch = ch.lower()
            if flag != 1 and len(word) >= 2:
                w = u''.join( word ).encode('utf-8', 'ignore')
                if len(w)> 3:
                    result.append( w )
                word = []
            word.append(ch)
            flag = 1
            pass
        elif is_unicode_hanzi( ch ):
            if flag != 2 and len(word) >= 2:
                w = u''.join( word ).encode('utf-8', 'ignore')
                if len(w)> 3:
                    result.append( w )
                word = []
            word.append(ch)
            flag = 2
        else:
            if len(word) >= 2:
                w = u''.join( word ).encode('utf-8', 'ignore')
                if len(w)> 3:
                    result.append( w )
            flag = 0
            word = []
    if len(word) >= 2:
        w = u''.join( word ).encode('utf-8', 'ignore')
        if len(w)> 3:
            result.append( w )
        
    return result

if __name__ == '__main__':
    import sys
    import re
    print full2half('abcd；def')
    sys.exit()
    #f = open('/letv/zoujiaqi/AlbumMatch/t.txt')
    f = open('/letv/zoujiaqi/search_and_rec/knowledge_graph/album_match/jb_seg/name.dict.utf8')
    for line in f:
       line = line.strip()
       if line == '': continue
       line = tradition2simple( line )
       print line
       continue
       line = line.replace(',', '|')
       line = line.replace('、', '|')
       line = line.replace('；', '|')
       line = line.replace(';', '|')
       line = line.replace('/', '|')
       line = line.replace('，', '|')
       line = line.replace('　', ' ')
       line = line.replace("'",'')
       words = line.split('|')
       for word in words:
           word = word.strip()
           if word =='': continue
           if word.isalnum(): continue
           if word.find('·') != -1: continue
           if word.find('.') != -1: continue
           if word.find(' ') != -1: continue
           if word.find('（') != -1: continue
           if word.find('&') != -1: continue
           if word.find('(') != -1: continue
           if re.search('[A-Za-z0-9]', word):
               continue
           if len( word) >= 15: continue
           print word
           
       #line = line.replace(',', '|')
       #print line 
       
    f.close()
    sys.exit()
    s = '''Matthew O'Callaghan;Todd Wilderman约翰·马洛瑞·亚瑟'''
    s = """张和涛  黄蕾 龚红玉李世超 龚红玉;李世超 约翰·马洛瑞·亚瑟 岑范，罗君雄 岑范;罗君雄 肯尼思·布拉纳 肯尼思·布莱纳格徐克Jean-Luc François肯尼思·布拉纳Paul Tanter约翰·马洛瑞·亚瑟 詹尼Paul;Tanter(小)徐克Sophie Aleet-Coche"""

    words = split_get_words(s )
    #print s, "###"
    #for w in words: print w

    #t = Test()
    #print t.cleanString('暗红三十六未删减版')
    #print t.cleanString('暗红36未删减版')
    print fromHanDigit('二十五'), 25

    print  is_english_line('Joe Russo·A')
    print  is_english_line('Joe Russo')
    print fromHanDigit('十')
    print fromHanDigit('十三')
    print fromHanDigit('五百十三')
    print fromHanDigit('三百六十五') == '365'
    print fromHanDigit('六十五') == '65'
    print fromHanDigit('五百零六') =='506'
    print fromHanDigit('一九一八') == '1918'
    print fromHanDigit('四千七百三十') == '4730'
    print fromHanDigit('五千零六十') == '5060'
    print fromHanDigit('两千零七') == '2007'
    print fromHanDigit('四千三百零九') == '4309'
    print fromHanDigit('一千零一') == '1001'

    sys.exit()
    res, _ = deal_country(u'【07泰影】爱你24小时不打烊')
    print res
    f = open('peo.dic', 'rb')
    for line in f:
        line = line.strip()
        if line == '': continue
        line = tradition2simple( line )
        pass
        result = format_peoplename( line )
        print line
        for r in result:
            print '\t', r
    f.close()
    pass

